from scrapy import Request

import scrapy
from scrapy.http import HtmlResponse

from spider2.items import MovieItem


class DoubanSpider(scrapy.Spider):
    name = 'douban'
    allowed_domains = ['movie.douban.com']

    def start_requests(self):
        for page in range(10):
            spider_url = f'https://movie.douban.com/top250?start={page * 25}&filter='
            yield scrapy.Request(spider_url, callback=self.parse)
            print(spider_url)

    # start_urls = ['https://movie.douban.com/top250']

    def parse(self, response):
        for list_item in response.xpath('//div[@class="article"]//li'):

            movie_item = MovieItem()
            movie_item['title'] = list_item.xpath('.//span[@class="title"][1]/text()').get('')  # 括号里面表示没有取到值就是空111
            # print(movie_item['title'])   # 我获取错误了 他获取的一长串
            movie_item['num'] = list_item.xpath(".//div[@class='item']/div[@class='info']/div[@class='bd']/div[@class='star']/span[4]").get()
            movie_item['href'] = list_item.xpath(".//div[@class='item']/div[@class='pic']/a/img/@src").get()

            movie_item['spider_rank'] = list_item.xpath(".//span[@class='rating_num']/text()").get()
            # print(movie_item['title'])
            movie_item['Introduction'] = list_item.xpath("string(.//span[@class='inq'])").get()
            print(movie_item['title'], movie_item['href'], movie_item['spider_rank'], movie_item['num'] ,movie_item['Introduction'])
            yield movie_item

        # href_list = sel.xpath("/div[@class='paginator']/a/@href")
        # for href in href_list:
        #
        #     url = response.urljoin(href.extract())
        #     print(url)
        # yield Request(url=url)

